{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.contrib as tc\n",
    "import gym\n",
    "from random import sample\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    }
   ],
   "source": [
    "gamma=0.99 \n",
    "tau=0.001 \n",
    "normalize_returns=False \n",
    "normalize_observations=True\n",
    "batch_size=128 \n",
    "observation_range=(0., 1000.) \n",
    "action_range=(-0.05, 0.05) \n",
    "return_range=(-np.inf, np.inf),\n",
    "critic_l2_reg=0. \n",
    "actor_lr=1e-4 #1e-5\n",
    "critic_lr=1e-4 #3e-5\n",
    "clip_norm=None \n",
    "reward_scale=1.\n",
    "\n",
    "env = gym.make(\"Pendulum-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.]\n",
      "[-2.]\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "Tensor(\"state_encode/input_1:0\", shape=(?, ?, 3), dtype=float32)\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, None, 1)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, None, 3)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, None, 4)      0           input_2[0][0]                    \n",
      "                                                                 input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       [(None, 64), (None,  13248       concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          16640       gru[0][1]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            257         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/mul (TensorF [(None, 1)]          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/truediv (Ten [(None, 1)]          0           tf_op_layer_policy/mul[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/add (TensorF [(None, 1)]          0           tf_op_layer_policy/truediv[0][0] \n",
      "==================================================================================================\n",
      "Total params: 30,145\n",
      "Trainable params: 30,145\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#state place holder\n",
    "obs_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0]\n",
    "action_high = env.action_space.high\n",
    "action_low = env.action_space.low\n",
    "print(action_high)\n",
    "print(action_low)\n",
    "\n",
    "#RNN network for state\n",
    "with tf.compat.v1.variable_scope(\"state_encode\"):\n",
    "    obs_input = tf.keras.layers.Input(shape = (None,obs_dim))\n",
    "    action_input = tf.keras.layers.Input(shape = (None,action_dim))\n",
    "    #obs_h = tf.keras.layers.Dense(4,activation = 'relu')(obs_input)\n",
    "    rnn_layer,state_h = tf.keras.layers.GRU(64,return_state=True)(tf.keras.layers.Concatenate()([action_input,obs_input]))\n",
    "    print(obs_input)\n",
    "\n",
    "#policy network\n",
    "with tf.compat.v1.variable_scope(\"policy\"):\n",
    "    #output action\n",
    "    action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(state_h)\n",
    "    action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_policy\"):\n",
    "    #output action\n",
    "    target_action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(state_h)\n",
    "    target_action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "\n",
    "    \n",
    "#output value function\n",
    "with tf.compat.v1.variable_scope(\"value_function\"):\n",
    "    value_h = tf.keras.layers.Dense(4,activation = 'relu')(state_h)\n",
    "    new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    value_input = tf.keras.layers.Concatenate()([new_action_input,value_h])\n",
    "    value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(value_input)\n",
    "    Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(value_h2)\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_value_function\"):\n",
    "    target_value_h = tf.keras.layers.Dense(4,activation = 'relu')(state_h)\n",
    "    target_new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    target_value_input = tf.keras.layers.Concatenate()([target_new_action_input, target_value_h])\n",
    "    target_value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(target_value_input)\n",
    "    target_Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(target_value_h2)\n",
    "\n",
    "#actor, critic model\n",
    "model = tf.keras.Model([obs_input,action_input,new_action_input],[action_output,Q_value_output])\n",
    "actor_model = tf.keras.Model([obs_input,action_input],action_output)\n",
    "critic_model = tf.keras.Model([obs_input,action_input,new_action_input],Q_value_output)\n",
    "target_actor_model = tf.keras.Model([obs_input,action_input],target_action_output)\n",
    "target_critic_model = tf.keras.Model([obs_input,action_input,target_new_action_input],target_Q_value_output)\n",
    "actor_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py:433: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
     ]
    }
   ],
   "source": [
    "#computation graph\n",
    "#policy gradient ascent\n",
    "Q_grad = tf.gradients(critic_model.output,new_action_input)\n",
    "Q_action_grad = tf.compat.v1.placeholder(tf.float32, shape=(None,action_dim))\n",
    "actor_model_weights = actor_model.trainable_weights\n",
    "action_grad = tf.gradients(actor_model.output,actor_model_weights,-Q_action_grad)\n",
    "act_grad_ascent = tf.train.AdamOptimizer(actor_lr).apply_gradients(zip(action_grad,actor_model_weights))\n",
    "\n",
    "#critic mse\n",
    "adam  = tf.keras.optimizers.Adam(critic_lr)\n",
    "critic_model.compile(loss=\"mse\", optimizer=adam)\n",
    "\n",
    "#target networks\n",
    "#moving average\n",
    "ema = tf.train.ExponentialMovingAverage(decay=0.99)\n",
    "policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"policy\")\n",
    "value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"value_function\")\n",
    "target_policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_policy\")\n",
    "target_value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_value_function\")\n",
    "target_policy_upd = ema.apply(policy_weights)\n",
    "target_value_upd = ema.apply(value_weights)\n",
    "target_policy_assign = tf.group([tf.assign(target_policy_weights[i], ema.average(policy_weights[i])) for i in range(len(policy_weights))])\n",
    "target_value_assign = tf.group([tf.assign(target_value_weights[i], ema.average(value_weights[i])) for i in range(len(value_weights))])\n",
    "#ema.average(critic_model_weights)\n",
    "\n",
    "\n",
    "#initialization\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:  0\n",
      "rewards:  -1271.9384737978355\n",
      "epoch:  10\n",
      "rewards:  -1252.597168035936\n",
      "epoch:  20\n",
      "rewards:  -1447.8358206954886\n",
      "epoch:  30\n",
      "rewards:  -1477.3626703833968\n",
      "epoch:  40\n",
      "rewards:  -1137.845791613746\n",
      "epoch:  50\n",
      "rewards:  -1182.1869937978938\n",
      "epoch:  60\n",
      "rewards:  -1350.9130344654234\n",
      "epoch:  70\n",
      "rewards:  -1047.5941090259587\n",
      "epoch:  80\n",
      "rewards:  -1282.3751260146157\n",
      "epoch:  90\n",
      "rewards:  -1276.3040482487634\n",
      "epoch:  100\n",
      "rewards:  -1455.5606115141368\n",
      "epoch:  110\n",
      "rewards:  -1270.3131475870225\n",
      "epoch:  120\n",
      "rewards:  -1128.0653021276848\n",
      "epoch:  130\n",
      "rewards:  -1391.2790307897708\n",
      "epoch:  140\n",
      "rewards:  -1327.95705664324\n",
      "epoch:  150\n",
      "rewards:  -974.4201983476329\n",
      "epoch:  160\n",
      "rewards:  -1238.4883300067652\n",
      "epoch:  170\n",
      "rewards:  -1295.111963592842\n",
      "epoch:  180\n",
      "rewards:  -1390.962132948963\n",
      "epoch:  190\n",
      "rewards:  -1213.3714018059516\n",
      "epoch:  200\n",
      "rewards:  -1270.3941955884904\n",
      "epoch:  210\n",
      "rewards:  -1164.2960169016683\n",
      "epoch:  220\n",
      "rewards:  -1302.3897521245124\n",
      "epoch:  230\n",
      "rewards:  -1426.2092187762717\n",
      "epoch:  240\n",
      "rewards:  -1336.3982135358665\n",
      "epoch:  250\n",
      "rewards:  -1345.4288542951306\n",
      "epoch:  260\n",
      "rewards:  -1242.3286710425648\n",
      "epoch:  270\n",
      "rewards:  -1178.3648204105289\n",
      "epoch:  280\n",
      "rewards:  -1156.0051444451392\n",
      "epoch:  290\n",
      "rewards:  -1081.1816967372365\n",
      "epoch:  300\n",
      "rewards:  -1035.526037843859\n",
      "epoch:  310\n",
      "rewards:  -1087.2540248188768\n",
      "epoch:  320\n",
      "rewards:  -1585.8117296320054\n",
      "epoch:  330\n",
      "rewards:  -1083.1800830083935\n",
      "epoch:  340\n",
      "rewards:  -1127.3783068263301\n",
      "epoch:  350\n",
      "rewards:  -1293.2283391102694\n",
      "epoch:  360\n",
      "rewards:  -1261.0518523197663\n",
      "epoch:  370\n",
      "rewards:  -1303.5023398709268\n",
      "epoch:  380\n",
      "rewards:  -1316.2061418712995\n",
      "epoch:  390\n",
      "rewards:  -1339.0067708235224\n",
      "epoch:  400\n",
      "rewards:  -1049.1509284571216\n",
      "epoch:  410\n",
      "rewards:  -1350.9312522327866\n",
      "epoch:  420\n",
      "rewards:  -1343.576741590823\n",
      "epoch:  430\n",
      "rewards:  -1283.5959362862463\n",
      "epoch:  440\n",
      "rewards:  -1357.1657937356474\n",
      "epoch:  450\n",
      "rewards:  -1056.0896679510006\n",
      "epoch:  460\n",
      "rewards:  -1088.0713120197006\n",
      "epoch:  470\n",
      "rewards:  -1455.8400320267644\n",
      "epoch:  480\n",
      "rewards:  -1281.7797130287915\n",
      "epoch:  490\n",
      "rewards:  -1181.7514098707106\n",
      "epoch:  500\n",
      "rewards:  -1291.3605333522623\n",
      "epoch:  510\n",
      "rewards:  -1123.433386536489\n",
      "epoch:  520\n",
      "rewards:  -1264.8498304655827\n",
      "epoch:  530\n",
      "rewards:  -923.3948869950218\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-d7a64369ec7f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    102\u001b[0m             target_actions = sess.run(target_action_output, feed_dict = {\n\u001b[1;32m    103\u001b[0m                 \u001b[0mobs_input\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnext_obs_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m                 \u001b[0maction_input\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnext_act_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m             })\n\u001b[1;32m    106\u001b[0m             target_Qs = sess.run(target_Q_value_output, feed_dict = {\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mfetch_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mmake_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mbuild_results\u001b[0;34m(self, session, tensor_values)\u001b[0m\n\u001b[1;32m    551\u001b[0m           \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed_handles\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m           \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feeds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    554\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m           \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensor_values\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "EPOCH = 10000\n",
    "max_step = 200\n",
    "max_seq = 1\n",
    "batch_size = 128\n",
    "epsilon = 0.9\n",
    "epsilon_decay = 0.99995\n",
    "\n",
    "replay_buffer = []\n",
    "max_size = 10000\n",
    "rewards_history = []\n",
    "\n",
    "for epoch in range(EPOCH):\n",
    "    #print(\"epoch: \",epoch)\n",
    "    obs_seq_list = []\n",
    "    act_seq_list = []\n",
    "    obs = env.reset()\n",
    "    obs = obs.reshape((obs_dim,))\n",
    "    obs_seq_list.append(obs)\n",
    "    act_seq_list.append(np.zeros(action_dim))\n",
    "    for t in range(max_step):\n",
    "        #interaction\n",
    "        obs_seq = np.asarray(obs_seq_list)\n",
    "        act_seq = np.asarray(act_seq_list)\n",
    "        obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "        act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        #pad input\n",
    "        pad_width = max_seq-np.size(obs_seq,1)\n",
    "        next_pad_width = max_seq-np.size(obs_seq,1)+1\n",
    "        obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        \n",
    "        #choose action\n",
    "        r = np.random.rand(1)\n",
    "        epsilon = epsilon*epsilon_decay\n",
    "        if r>epsilon:\n",
    "            action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        \n",
    "        action = action.reshape((action_dim,))\n",
    "        next_obs,reward,done,_ = env.step(action)\n",
    "        next_obs = next_obs.reshape((obs_dim,))\n",
    "        \n",
    "        obs_seq_list.append(next_obs)\n",
    "        act_seq_list.append(action)\n",
    "        #keep the length same\n",
    "        if len(obs_seq_list)>max_seq:\n",
    "            obs_seq_list.pop(0)\n",
    "            act_seq_list.pop(0)\n",
    "        next_obs_seq = np.asarray(obs_seq_list)\n",
    "        next_act_seq = np.asarray(act_seq_list)\n",
    "        next_obs_seq = next_obs_seq.reshape((1,-1,obs_dim))\n",
    "        next_act_seq = next_act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        #pad data\n",
    "        pad_width = max_seq-np.size(obs_seq,1)\n",
    "        next_pad_width = max_seq-np.size(next_obs_seq,1)\n",
    "        obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        next_obs_seq = np.pad(next_obs_seq,((0,0),(next_pad_width,0),(0,0)))\n",
    "        next_act_seq = np.pad(next_act_seq,((0,0),(next_pad_width,0),(0,0)))\n",
    "        \n",
    "        #record\n",
    "        replay_buffer.append([obs_seq,act_seq,action,reward,next_obs_seq,next_act_seq,done])\n",
    "        if len(replay_buffer)>max_size:\n",
    "            replay_buffer.pop(0)\n",
    "        \n",
    "        #move to the next\n",
    "        obs = next_obs\n",
    "        \n",
    "        \n",
    "        #train\n",
    "        #get data from the buffer\n",
    "        if len(replay_buffer)>batch_size and t%5==0 and t!=0:\n",
    "            train_datas = sample(replay_buffer,batch_size)\n",
    "            array = np.asarray(train_datas)\n",
    "            obs_seq_data = np.stack(array[:,0]).reshape((batch_size,max_seq,-1))\n",
    "            act_seq_data = np.stack(array[:,1]).reshape((batch_size,max_seq,-1))\n",
    "            action_data = np.stack(array[:,2]).reshape((batch_size,-1))\n",
    "            reward_data = np.stack(array[:,3]).reshape((batch_size,-1))\n",
    "            next_obs_data = np.stack(array[:,4]).reshape((batch_size,max_seq,-1))\n",
    "            next_act_data = np.stack(array[:,5]).reshape((batch_size,max_seq,-1))\n",
    "            done_data = np.stack(array[:,6]).reshape((batch_size,-1))\n",
    "\n",
    "            #train actor\n",
    "            new_action = sess.run(action_output, feed_dict={\n",
    "                obs_input:  obs_seq_data,\n",
    "                action_input: act_seq_data\n",
    "            })\n",
    "            Q_action_grads = sess.run(Q_grad, feed_dict={\n",
    "                obs_input:  obs_seq_data,\n",
    "                action_input: act_seq_data,\n",
    "                new_action_input: new_action\n",
    "            })[0]\n",
    "            sess.run(act_grad_ascent, feed_dict={\n",
    "                obs_input:  obs_seq_data,\n",
    "                action_input: act_seq_data,\n",
    "                Q_action_grad:Q_action_grads\n",
    "            })\n",
    "            #train critic\n",
    "            target_actions = sess.run(target_action_output, feed_dict = {\n",
    "                obs_input: next_obs_data,\n",
    "                action_input: next_act_data,\n",
    "            })\n",
    "            target_Qs = sess.run(target_Q_value_output, feed_dict = {\n",
    "                obs_input: next_obs_data,\n",
    "                action_input: next_act_data,\n",
    "                target_new_action_input: target_actions\n",
    "            })\n",
    "            Q_y = reward_data + gamma*target_Qs*(1-done)\n",
    "            evaluation = critic_model.fit([obs_seq_data,act_seq_data,action_data], Q_y, verbose=0)\n",
    "#             if t==max_step-5:\n",
    "#                 print(t,evaluation.history)\n",
    "            #update target\n",
    "            sess.run(target_policy_upd)\n",
    "            sess.run(target_value_upd)\n",
    "            sess.run(target_policy_assign)\n",
    "            sess.run(target_value_assign)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "                \n",
    "    #evaluate\n",
    "    if epoch%10==0:\n",
    "        obs_seq_list = []\n",
    "        act_seq_list = []\n",
    "        ave_rewards = 0\n",
    "        for times in range(5):\n",
    "            rewards = 0\n",
    "            obs = env.reset()          \n",
    "            for t in range(500):\n",
    "                #interaction\n",
    "                obs_seq = np.asarray(obs_seq_list)\n",
    "                act_seq = np.asarray(act_seq_list)\n",
    "                obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "                act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "                #pad input\n",
    "                pad_width = max_seq-np.size(obs_seq,1)\n",
    "                obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "                act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "                #choose action\n",
    "                action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "                action = action.reshape((action_dim,))\n",
    "                next_obs,reward,done,_ = env.step(action)\n",
    "                obs = next_obs.reshape((obs_dim,))\n",
    "                obs_seq_list.append(obs)   \n",
    "                act_seq_list.append(action)\n",
    "                if len(obs_seq_list)>max_seq:\n",
    "                    obs_seq_list.pop(0)\n",
    "                    act_seq_list.pop(0)\n",
    "\n",
    "                rewards = rewards+reward\n",
    "                if done:\n",
    "                    break\n",
    "            ave_rewards = ave_rewards+rewards\n",
    "            \n",
    "        ave_rewards = ave_rewards/5\n",
    "        rewards_history.append(ave_rewards)\n",
    "        print(\"epoch: \",epoch)\n",
    "        print(\"rewards: \", ave_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [0,1,2,3]\n",
    "a.append(4)\n",
    "print(a)\n",
    "a.pop(0)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
